{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A General Framework for Linear Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sympy\n",
    "from sympy import init_printing\n",
    "import numpy as np\n",
    "from numpy.linalg import inv\n",
    "\n",
    "init_printing()\n",
    "np.set_printoptions(precision=3, suppress=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A simple enlightment on the fact that\n",
    "$$\n",
    "    \\Phi^T \\Phi = \\sum_{i=1}^{N} \\phi(x_i) \\phi(x_i)^T\n",
    "$$\n",
    "\n",
    "Consider\n",
    "* $\\Phi \\in \\mathbb{R}^{N\\times M}$\n",
    "* $\\phi(x_i) \\in \\mathbb{R}^{M\\times 1}$\n",
    "\n",
    "Where\n",
    "$$\n",
    "    \\Phi = \\begin{bmatrix} \\phi(x_1)^T \\\\ \\vdots \\\\ \\phi(x_N)^T \\end{bmatrix}\n",
    "$$\n",
    "\n",
    "Thus,\n",
    "\n",
    "$$\n",
    "    \\Phi^T\\Phi = \\begin{bmatrix} \\phi(x_1) & \\ldots & \\phi(x_N) \\end{bmatrix} \\begin{bmatrix} \\phi(x_1)^T \\\\ \\vdots \\\\ \\phi(x_N)^T \\end{bmatrix} = \\sum_{i=1}^{N} \\phi(x_i) \\phi(x_i)^T\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### An Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "phi_01 = sympy.symbols(\"\\phi_{0}(x_1)\")\n",
    "phi_02 = sympy.symbols(\"\\phi_{0}(x_2)\")\n",
    "\n",
    "phi_11 = sympy.symbols(\"\\phi_{1}(x_1)\")\n",
    "phi_12 = sympy.symbols(\"\\phi_{1}(x_2)\")\n",
    "\n",
    "phi_21 = sympy.symbols(\"\\phi_{2}(x_1)\")\n",
    "phi_22 = sympy.symbols(\"\\phi_{2}(x_2)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "Phi = sympy.Matrix([\n",
    "    [phi_01, phi_02],\n",
    "    [phi_11, phi_12],\n",
    "    [phi_21, phi_22]\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJsAAABLCAMAAABQgQWRAAAAPFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAo1xBWAAAAE3RSTlMAMquZdlQQQOkwRCKJZs273e9sxmG/9gAAAAlwSFlzAAAOxAAADsQBlSsOGwAABIhJREFUaAXtWtu2oyAMxevMVK12/P9/nYRLEi4iejhrzoM8tCC42U0CYS+qml2XVv2cshlKSjV710MZfg41tSChdkdujUdr8lq6MS3xM/1kSQwOhyaGFOEtCW5zCK6mNXrkHrSJmV2f/b6LV8btfUxgfAdM4mbMrQwvxa0P4ZcufCLaczRcdOpqNKAQL8Ht5UcfwG/HZlNq/IRcgvZtvAS3yAXTFszmN98vvx22buOF3JrX1E2Db6feunSEZT29+tCJc87jX8ELuHWDwuWNsQoc7C9ezQYyQnPZerXp6G9oDSxUCy2mVDHe3K3daN4nPJ8bssAda27VAkwGkyo2syvP8O68jwqt2nQtOXqgWsStGG8GzNb+RsLzuC04Cxqr29SGUQRMoHxMQGH9TRsdIcDWjYNSpRwPV1uzwwcUwvO4rRg5yO3djjpXfLTFLDd8b6f9oIRbOR5aYtqNDdLcPuBSDLdpH8yP2HSkWZ8CNZHdmNvr0KdX8CCCjJcU4Xl2e4NVMNzg9zZ6IEQ+FLsWhlH16L5eO5q5UeziWK9cwYNd1PqE8DxuyBhciq4wdvvo4WaTeO2DWoGbTdPMrcMoSJYreIq2IsLzuMHyW9a+w7xg4s1EgDHy2Pb92HeLdjN4gDz5jvIIEb2Ax3mM8HxuZBWwMK/TZM4ibvmsQYehE7wXGN8kN8YLudF5DJOB3d8U/yayiBpcGg3TBA/BWiFe8x6GYdWBzHghN46duZvdTq1Ws/PwtFO37a0OxpMzUiHeR5/BEV/ghdxsNDELXcucLSPa/qtfwQu5+cjUorChJ7ZCPgs78u0ivEJu+Zm+qffhds+wj93q2S3czAC5aF0dMbiLl/Ip75dutsz+9mhnZyRlz1HUTuZT6uX8R4+Cyl28hE9va92Akmvexktwi8KNTy1uOu/70c6e1lWkjuto5zzeJe0s1HEV7ZzHu6adhSKoop3zeIfa+dfvPya4hdaFB6QIamjnE7ykdv77WyhioXUlVg3tfIoHtijVzhKrhnY+xbugnSVWDe18indBO0usGtr5FI915rl2FmuhinY+wbuknQHLqeNkrqfefK4XOozeSOBd086sjsEdkQjlXqF1YWBUKDnzGzAmwnu0c2S55IOiM37ijJQE+x8PH273rP7YrZ7d7mrdIwZ38VI+pf2SJnu0szaFyDMpu93VumTmoHIXL8HtttYNKLnmbbwEtyjcHu3szMwH4MBu4p6Y1Wwd7ZzH416VvmvTN354GoR7Z6Fmq2jnPJ7oZe3p2c27JwaK9ia4inYW6jiBJ3phVutej5vUzkLNVtHOeTzRe8DN186kZitp5zwe9Z7fO2uz2pvgStqZ1HESj3oP1oK4J0Zu7ia4knbO41GvOr13BmqkZitp5zwe9aoC7cxqto52zuNxL2chb52isdxhS6hZtiB62hZSwoXaOY8nehkv5EbJlNUskIm0LithcaZxtOV3IR7PJvBCbl+5J5acXP0reCE3hxl8F2nd4J1cswivkFtunm/rM9x+7n+P4f+AWMy/qL7NCpeA9X+P+179A9k3UuIvLmgGAAAAAElFTkSuQmCC\n",
      "text/latex": [
       "$$\\left[\\begin{matrix}\\phi_{0}(x_1) & \\phi_{0}(x_2)\\\\\\phi_{1}(x_1) & \\phi_{1}(x_2)\\\\\\phi_{2}(x_1) & \\phi_{2}(x_2)\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡\\phi_{0}(x_1)  \\phi_{0}(x_2)⎤\n",
       "⎢                            ⎥\n",
       "⎢\\phi_{1}(x_1)  \\phi_{1}(x_2)⎥\n",
       "⎢                            ⎥\n",
       "⎣\\phi_{2}(x_1)  \\phi_{2}(x_2)⎦"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# N: number of datapoints\n",
    "# M: number of basis functions\n",
    "Phi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0oAAAAzBAMAAABbMeVKAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMA74lUMhAidkSZZs27q92TbWBvAAAACXBIWXMAAA7EAAAOxAGVKw4bAAANCUlEQVR4Ae1cTYglVxU+/Vfd87rnTSCCoovURkMjkg7RuPGnAxEMWXQrkixELHBjGkmagRBmITYSdDc2o3ERNDxcDIyR0KCMA23kbYzMbJyNk4UgLYGZEBeOZhKzkfb83HN/qk7frtv9JouhL6TqVt1zv/Odc1699/q8LwMwkTH9i5dKcArNl6+ul6Dfg7YP7P9nAlF9Cv5WglJmPt2c3ilBv8dsH9r/N9z/+FcmENVNWGsKYMrMh83s+wXg95rpNx5/GD4ykaDehue2CoDKzIdb1QcF4Pee6WNFVWo0AdVIZ3i+yPMvdz86GjUqNN/QfXgeMMjMe9Gt3tPKW3YRARiYLAx2fiNPGr20LPNOdGeKE9Nx6QOPkziR8MuqVKvT7+mEzqcbOt6iQzpqvSwzr8a6j8436DDcpGPpmNYN1VhndGZEgFrvGex0Sc5Zy7yTBEhxqnF8W9IHHiehI2SPWKXLsZs5+mif3o5vybzWW2Xmi43uo/M36bBMh+LhAzcQoyp12bU81XptWead6E4+K05Kh9OHCVTTxImEX1YlfUUP0pI8iA4uqZPP6wTgiOYeipGmEaXlL3jIz+Z1uYuIKxl2tG9G1w+xzDsBMPKR0gFKH4DitGKl8KGoSgurBIdj2PBJD18DWKh1MbDSO6XmNxWXz3P3AfwQXkzu9bzwm7qIyHjVoXSD4YVQpbxl1gkiGflI6QCmD4fitOhQ+GVV8g/ls4zrD4tb8OQjX1iX68DqiObVngfmyRWY/e0jf0rv9buqnVkXERcy7HhbqFLeMusEkbr5aNPB9OGo6YCjlVu4gvdKnqWvPvOdBrfgOIf/Pf39756lpxEHfiY9sL8v88DqqObz2wBzL9z4sbwR0GfS1P7+bYdecJr90c8fFfMuIkCOHe/yVcpa5p0QkK+Sx2nToY/0gNPKLX8kF1Tp1MoGXIazZ9HzNcxjPbg13IHZHbwc0EGHshJzXu9vTq2gpRHA1+G1zX8CPEmgrypy4fmn1WixmX+wBgvRDIYbURqMVikfBzthnoG2tL+UdpoPK0DyyDi8MSRL+mKEU1Cla/jN/jODeuY6wFv4/KxPvzfTzP5yD0Fm6KBDWbG5rPc251bQcAXgk/DQ+qPwxN8J9EuKXHZeGg2a6eu/gi/ix2gX0QpGGlEajFYpG4c4YZ7BibS/lHaSDzNA9Cg4vNEny/XFCKd/lQZ7+Nb5ynAL8A/Mj+GzBItj3M8xzd7WBH56d/cfu7uv46WYy3pvc24FTW0BrMMfCJIjXaNZ+XgNP3lO7b0NZ9ahi2gGI40oDmZ6d/ePn9jdxRdkPg5xwjyDE2l/Me12PswA0aPg8EafLNcXI5z+VVraxiq9/u1VuJ+rBHBmE/e3qoR33GtHzH2V+plzK4jCBfgvHY5TpatYpcW/3oGpFa5SimgGI40ofcm5ZykfBzu57qskTqT9pS+uJB9mgOhRcHgjVkmS5fpiZVWaGlWj6t21Bt7hdzyAtVWE4yot7OFMh2Ml5rKOD3FPc8BWEL11wOwd2sNVOk+z8vFx2IDnPvs+DEcGoh0MeQcNxlUpHwc7WWGeEW2gbpnSTvNhBYgeHQ5tDMliOozT/1la3Bw0S9tr69QKwk+4mfUfwPy6VGGwEyXRsRJzWe9vTq0g+hj+3PxteN5VST+GIx99pr/Hj9Grs1h0C9EOhhtRGoyrUj4OdiI8I9rcLVPaaT4sOujR4aS5ZTr85al/lRb2pvHBxGfpo9yw+cnKX/DFKlU4VUdpc6zEXNaXAXqa0/dO/GK68O7S7WrkqnQhAi+Ynod6aRt/8ZjaNBDtYPhbrwbjqpSPg50Iz4g2fbUGpZ3mwwoQPToc2hiSxXQYp3+V4InfnFsF/Fz6gJ/mb73w9MuYR37HW1yNsudYibms48Pf05zaI/gGMPfm8+d+jZj8jve7CLxgOvvyn38Gc3fgzJaFaAbDzRkNxlUpHwc7EZ4R7UtEU2mn+bACRI8OhzaGZEmviHAKqkSPDgz5O96wwSkPrtKzekVnZcXmUqX+5twKCg0UqlL7L/XYV3ZODyPwdzwwEK1g2LsGo1XKx8FO5NXknXC3zNNO82EEyN0GxuGNIVlsyzglVbqIUc9v099LUmXK0cx9eNAuAd2Ap/iIBzKX9d7m0grilyKjPIxHbjfyVeFh0OCGc/T3UmgFg0c0ghHvGswcbaeRjYOdMGpwwt0yTzvJhxUgp49xeKNPltgyTkmVauK8vLyOxzdoiqN65X+bMLcjF61jrev9zaUVtNQ4pGf+9ab75cLdKDpx+236wku4yUCsCSsNhr13gyFLjtOKg50wz+CEu2X8gwv58KPGmRUge2QcabNpbsWWcUqqNPIOIfmlyucgrNNsFC7LzKtx2BkyE9/rNR8Eq2oc5g5xFO4cwi5rmXcSfOAs4FRjvPSD0xfhJHTkeSipksfF19YouuC3hOi6Oy00p88MHfKWoldHPWcR+7PLW2adJMxjS3lHjZYTJxL+EasUgZ5M734GTqqUyfHd1XZKyzvjPiydVCnkojO7q9pO1/LuOLVunFTJyoq7d1e1na7lnXEflk6qFHLRmd1VbadreXecWjd8lRpdtb5/VLpofP/QJTl7S/mT3S3qF7VGjRMc6zti1jLvRH30PXtXRuRdKWjOt6HtLDQ/gLOvUu0MqrGb8Ol0wyevz0i/y9+ITWXuLatxvOgsa72X4DgnusbnrGXeSYLT40JdVePYWEhh/781cr6x/d4ehebt7e66U6ViRV8K7FmlOO4v8VqNL+uEzt0/94OiBpUW0XCWeSeRfa9p7axSxtIR2O4g5Hwvd6yDUilF53wY5l0AvuOrpC+DS6mhdLXm3U3fY5Jr7VXN6Oag/QtdLTZ1lmrXwtHWWYSTtVQ6tpM0gMOv1JURub/lW6Ze3Gj49lFFcWSoevPDKWqVFlad7c10z+GKPrSPWL2o21Mc1v59aEJF5dDrnInckoJmAuQeNvnslw9vfjhNrZI+yL7h7rYubtGkpgMO7erLFbCiD+cRq9ottXGu0H110sYRJzFO3jLrxBHofVJXbcZIypKCHuxbetjkt1c+gvnhVF2VjqHoi1kF7d/8Nn7eJNLHD1GoeHjYwSIXeVcKmgtQetiE7KvUzzyQOWgmVQoawqVRKlkkFUCQH17D1ItaMij6YlaWhtBLH035IeO0pAYglrwUPM4iE9ZnBoFhIJsKFQ8K1rhvRS5gSoo36eeSFWDXt6/SpKhKlYKGcLiSShbxt9hIfviWqiUjRR9GoaxMDaGXPlryQ8HhH3wjHLaUJe/R6zMjgWEgmwoVjWocdMuKXMCUFO90VTID7PpO88Hrx6PKVRoEQWTQ/snPxKgVi+SHlqIvCAydJW9s46zZ8sOMUFGWvEf5bd7T4eZNcBIJFQ+qh3nfjFzA0JUbQfp4mEiSNnTyMQGqXKVIQ0iBxwJD5BrJDzFnHUUf3tHXjqkhlGqvgSk/zAgVZcl79FWKBIaBbCpUJJb9hhm5gIUqIZR7lswAu77TfPD68ahylSINIT2ZsWRxwSn6RH6I7z9O/hgUfXhHWdkaQid9tOWHGaEiLwWP/A6kdJAlNm8isolQkVj2G3bkDIauwnBVsgPs+G7l4/hUuUqRhpA+kGPJ4sAp+kR+iJ/lTi0ZFH0YirKyNYRcpVfBlh9mhIq8FDxylZQOOr0l/y+FI5sIFUN2D5vZkTMYugrDVckOsOO7lY/jU+UqRRrCSPvH2T3lFH0iP1z28seg6MNQlJWtIWScCyiKI9WlvHG1cNAJD4fjLNlFsOQqKR18+9/m/1RfmQgVBa3P0Y6cwZQUw7gq2QF2fLfycXyqXKVIEInPeSJZXHSKPpEfnvfyR2lvOGWgsrI1hFwltGSholSphYNOeCiOWLKLYMlVUjoi3orIXiIAFSoKWq9jUE+2wZQUw7gq2QF2fGsckRayjV5GVaoUCSJDZ4ezS92GaoQHlh9aij5cU1ZOtMIbUxz+w34DTaVKLRxyQsPjsCV3UIIlV8nTkeaNd5IKFQWt35FcDVkKmoKl/RFXJTPArm8fB2duAlRdlS4iVxFE8guDAySBYVD0sfzQNwgjRR8azTVkimPQ0JE3pjjcbSUnLRml4Gi31eOQpSx5j0GfyU6keeOdpEJF8tN3kKtW5IKtpBjoKQdnBdj17eOYFFVXpZpYsIZwqaEpDpYCztFHKHe6RH74Bi+p+q+jDGTLloYwSB9r3NySH2aEiq7hoh6DPpOdSPPGkz1AqOjY5k5EqhU5g3HknY1WgBnfk6LqqjTyfKqxn+KE0zAId5Jf77zC1a9HltXY38UJ53oU7iQ4Ptd+PWuZd+Ixek6Cq2ocb+mSotVC34Xmsf9k7qoU3duI5n0UfbF5PI9x5I0iWuU3bL2+qBPrnLfMOrHgcvdisE7k3Y2xeSfAY5snAI9N6F9eS0BPLiaaAfqX1ybzrxhOlNYJWJIB/FcM/w8fy/BiXNRKCQAAAABJRU5ErkJggg==\n",
      "text/latex": [
       "$$\\left[\\begin{matrix}\\phi_{0}(x_1)^{2} + \\phi_{1}(x_1)^{2} + \\phi_{2}(x_1)^{2} & \\phi_{0}(x_1) \\phi_{0}(x_2) + \\phi_{1}(x_1) \\phi_{1}(x_2) + \\phi_{2}(x_1) \\phi_{2}(x_2)\\\\\\phi_{0}(x_1) \\phi_{0}(x_2) + \\phi_{1}(x_1) \\phi_{1}(x_2) + \\phi_{2}(x_1) \\phi_{2}(x_2) & \\phi_{0}(x_2)^{2} + \\phi_{1}(x_2)^{2} + \\phi_{2}(x_2)^{2}\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡                                2                2                2          \n",
       "⎢                   \\phi_{0}(x_1)  + \\phi_{1}(x_1)  + \\phi_{2}(x_1)           \n",
       "⎢                                                                             \n",
       "⎢                                                                             \n",
       "⎣\\phi_{0}(x_1)⋅\\phi_{0}(x_2) + \\phi_{1}(x_1)⋅\\phi_{1}(x_2) + \\phi_{2}(x_1)⋅\\ph\n",
       "\n",
       "                                                                              \n",
       "            \\phi_{0}(x_1)⋅\\phi_{0}(x_2) + \\phi_{1}(x_1)⋅\\phi_{1}(x_2) + \\phi_{\n",
       "                                                                              \n",
       "                                            2                2                \n",
       "i_{2}(x_2)                     \\phi_{0}(x_2)  + \\phi_{1}(x_2)  + \\phi_{2}(x_2)\n",
       "\n",
       "                     ⎤\n",
       "2}(x_1)⋅\\phi_{2}(x_2)⎥\n",
       "                     ⎥\n",
       "2                    ⎥\n",
       "                     ⎦"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "E1 = Phi.T @ Phi\n",
    "E1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "phi_0 = sympy.Matrix([[phi_01, phi_02]]).T\n",
    "phi_1 = sympy.Matrix([[phi_11, phi_12]]).T\n",
    "phi_2 = sympy.Matrix([[phi_21, phi_22]]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0oAAAAzBAMAAABbMeVKAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMA74lUMhAidkSZZs27q92TbWBvAAAACXBIWXMAAA7EAAAOxAGVKw4bAAANCUlEQVR4Ae1cTYglVxU+/Vfd87rnTSCCoovURkMjkg7RuPGnAxEMWXQrkixELHBjGkmagRBmITYSdDc2o3ERNDxcDIyR0KCMA23kbYzMbJyNk4UgLYGZEBeOZhKzkfb83HN/qk7frtv9JouhL6TqVt1zv/Odc1699/q8LwMwkTH9i5dKcArNl6+ul6Dfg7YP7P9nAlF9Cv5WglJmPt2c3ilBv8dsH9r/N9z/+FcmENVNWGsKYMrMh83s+wXg95rpNx5/GD4ykaDehue2CoDKzIdb1QcF4Pee6WNFVWo0AdVIZ3i+yPMvdz86GjUqNN/QfXgeMMjMe9Gt3tPKW3YRARiYLAx2fiNPGr20LPNOdGeKE9Nx6QOPkziR8MuqVKvT7+mEzqcbOt6iQzpqvSwzr8a6j8436DDcpGPpmNYN1VhndGZEgFrvGex0Sc5Zy7yTBEhxqnF8W9IHHiehI2SPWKXLsZs5+mif3o5vybzWW2Xmi43uo/M36bBMh+LhAzcQoyp12bU81XptWead6E4+K05Kh9OHCVTTxImEX1YlfUUP0pI8iA4uqZPP6wTgiOYeipGmEaXlL3jIz+Z1uYuIKxl2tG9G1w+xzDsBMPKR0gFKH4DitGKl8KGoSgurBIdj2PBJD18DWKh1MbDSO6XmNxWXz3P3AfwQXkzu9bzwm7qIyHjVoXSD4YVQpbxl1gkiGflI6QCmD4fitOhQ+GVV8g/ls4zrD4tb8OQjX1iX68DqiObVngfmyRWY/e0jf0rv9buqnVkXERcy7HhbqFLeMusEkbr5aNPB9OGo6YCjlVu4gvdKnqWvPvOdBrfgOIf/Pf39756lpxEHfiY9sL8v88DqqObz2wBzL9z4sbwR0GfS1P7+bYdecJr90c8fFfMuIkCOHe/yVcpa5p0QkK+Sx2nToY/0gNPKLX8kF1Tp1MoGXIazZ9HzNcxjPbg13IHZHbwc0EGHshJzXu9vTq2gpRHA1+G1zX8CPEmgrypy4fmn1WixmX+wBgvRDIYbURqMVikfBzthnoG2tL+UdpoPK0DyyDi8MSRL+mKEU1Cla/jN/jODeuY6wFv4/KxPvzfTzP5yD0Fm6KBDWbG5rPc251bQcAXgk/DQ+qPwxN8J9EuKXHZeGg2a6eu/gi/ix2gX0QpGGlEajFYpG4c4YZ7BibS/lHaSDzNA9Cg4vNEny/XFCKd/lQZ7+Nb5ynAL8A/Mj+GzBItj3M8xzd7WBH56d/cfu7uv46WYy3pvc24FTW0BrMMfCJIjXaNZ+XgNP3lO7b0NZ9ahi2gGI40oDmZ6d/ePn9jdxRdkPg5xwjyDE2l/Me12PswA0aPg8EafLNcXI5z+VVraxiq9/u1VuJ+rBHBmE/e3qoR33GtHzH2V+plzK4jCBfgvHY5TpatYpcW/3oGpFa5SimgGI40ofcm5ZykfBzu57qskTqT9pS+uJB9mgOhRcHgjVkmS5fpiZVWaGlWj6t21Bt7hdzyAtVWE4yot7OFMh2Ml5rKOD3FPc8BWEL11wOwd2sNVOk+z8vFx2IDnPvs+DEcGoh0MeQcNxlUpHwc7WWGeEW2gbpnSTvNhBYgeHQ5tDMliOozT/1la3Bw0S9tr69QKwk+4mfUfwPy6VGGwEyXRsRJzWe9vTq0g+hj+3PxteN5VST+GIx99pr/Hj9Grs1h0C9EOhhtRGoyrUj4OdiI8I9rcLVPaaT4sOujR4aS5ZTr85al/lRb2pvHBxGfpo9yw+cnKX/DFKlU4VUdpc6zEXNaXAXqa0/dO/GK68O7S7WrkqnQhAi+Ynod6aRt/8ZjaNBDtYPhbrwbjqpSPg50Iz4g2fbUGpZ3mwwoQPToc2hiSxXQYp3+V4InfnFsF/Fz6gJ/mb73w9MuYR37HW1yNsudYibms48Pf05zaI/gGMPfm8+d+jZj8jve7CLxgOvvyn38Gc3fgzJaFaAbDzRkNxlUpHwc7EZ4R7UtEU2mn+bACRI8OhzaGZEmviHAKqkSPDgz5O96wwSkPrtKzekVnZcXmUqX+5twKCg0UqlL7L/XYV3ZODyPwdzwwEK1g2LsGo1XKx8FO5NXknXC3zNNO82EEyN0GxuGNIVlsyzglVbqIUc9v099LUmXK0cx9eNAuAd2Ap/iIBzKX9d7m0grilyKjPIxHbjfyVeFh0OCGc/T3UmgFg0c0ghHvGswcbaeRjYOdMGpwwt0yTzvJhxUgp49xeKNPltgyTkmVauK8vLyOxzdoiqN65X+bMLcjF61jrev9zaUVtNQ4pGf+9ab75cLdKDpx+236wku4yUCsCSsNhr13gyFLjtOKg50wz+CEu2X8gwv58KPGmRUge2QcabNpbsWWcUqqNPIOIfmlyucgrNNsFC7LzKtx2BkyE9/rNR8Eq2oc5g5xFO4cwi5rmXcSfOAs4FRjvPSD0xfhJHTkeSipksfF19YouuC3hOi6Oy00p88MHfKWoldHPWcR+7PLW2adJMxjS3lHjZYTJxL+EasUgZ5M734GTqqUyfHd1XZKyzvjPiydVCnkojO7q9pO1/LuOLVunFTJyoq7d1e1na7lnXEflk6qFHLRmd1VbadreXecWjd8lRpdtb5/VLpofP/QJTl7S/mT3S3qF7VGjRMc6zti1jLvRH30PXtXRuRdKWjOt6HtLDQ/gLOvUu0MqrGb8Ol0wyevz0i/y9+ITWXuLatxvOgsa72X4DgnusbnrGXeSYLT40JdVePYWEhh/781cr6x/d4ehebt7e66U6ViRV8K7FmlOO4v8VqNL+uEzt0/94OiBpUW0XCWeSeRfa9p7axSxtIR2O4g5Hwvd6yDUilF53wY5l0AvuOrpC+DS6mhdLXm3U3fY5Jr7VXN6Oag/QtdLTZ1lmrXwtHWWYSTtVQ6tpM0gMOv1JURub/lW6Ze3Gj49lFFcWSoevPDKWqVFlad7c10z+GKPrSPWL2o21Mc1v59aEJF5dDrnInckoJmAuQeNvnslw9vfjhNrZI+yL7h7rYubtGkpgMO7erLFbCiD+cRq9ottXGu0H110sYRJzFO3jLrxBHofVJXbcZIypKCHuxbetjkt1c+gvnhVF2VjqHoi1kF7d/8Nn7eJNLHD1GoeHjYwSIXeVcKmgtQetiE7KvUzzyQOWgmVQoawqVRKlkkFUCQH17D1ItaMij6YlaWhtBLH035IeO0pAYglrwUPM4iE9ZnBoFhIJsKFQ8K1rhvRS5gSoo36eeSFWDXt6/SpKhKlYKGcLiSShbxt9hIfviWqiUjRR9GoaxMDaGXPlryQ8HhH3wjHLaUJe/R6zMjgWEgmwoVjWocdMuKXMCUFO90VTID7PpO88Hrx6PKVRoEQWTQ/snPxKgVi+SHlqIvCAydJW9s46zZ8sOMUFGWvEf5bd7T4eZNcBIJFQ+qh3nfjFzA0JUbQfp4mEiSNnTyMQGqXKVIQ0iBxwJD5BrJDzFnHUUf3tHXjqkhlGqvgSk/zAgVZcl79FWKBIaBbCpUJJb9hhm5gIUqIZR7lswAu77TfPD68ahylSINIT2ZsWRxwSn6RH6I7z9O/hgUfXhHWdkaQid9tOWHGaEiLwWP/A6kdJAlNm8isolQkVj2G3bkDIauwnBVsgPs+G7l4/hUuUqRhpA+kGPJ4sAp+kR+iJ/lTi0ZFH0YirKyNYRcpVfBlh9mhIq8FDxylZQOOr0l/y+FI5sIFUN2D5vZkTMYugrDVckOsOO7lY/jU+UqRRrCSPvH2T3lFH0iP1z28seg6MNQlJWtIWScCyiKI9WlvHG1cNAJD4fjLNlFsOQqKR18+9/m/1RfmQgVBa3P0Y6cwZQUw7gq2QF2fLfycXyqXKVIEInPeSJZXHSKPpEfnvfyR2lvOGWgsrI1hFwltGSholSphYNOeCiOWLKLYMlVUjoi3orIXiIAFSoKWq9jUE+2wZQUw7gq2QF2fGsckRayjV5GVaoUCSJDZ4ezS92GaoQHlh9aij5cU1ZOtMIbUxz+w34DTaVKLRxyQsPjsCV3UIIlV8nTkeaNd5IKFQWt35FcDVkKmoKl/RFXJTPArm8fB2duAlRdlS4iVxFE8guDAySBYVD0sfzQNwgjRR8azTVkimPQ0JE3pjjcbSUnLRml4Gi31eOQpSx5j0GfyU6keeOdpEJF8tN3kKtW5IKtpBjoKQdnBdj17eOYFFVXpZpYsIZwqaEpDpYCztFHKHe6RH74Bi+p+q+jDGTLloYwSB9r3NySH2aEiq7hoh6DPpOdSPPGkz1AqOjY5k5EqhU5g3HknY1WgBnfk6LqqjTyfKqxn+KE0zAId5Jf77zC1a9HltXY38UJ53oU7iQ4Ptd+PWuZd+Ixek6Cq2ocb+mSotVC34Xmsf9k7qoU3duI5n0UfbF5PI9x5I0iWuU3bL2+qBPrnLfMOrHgcvdisE7k3Y2xeSfAY5snAI9N6F9eS0BPLiaaAfqX1ybzrxhOlNYJWJIB/FcM/w8fy/BiXNRKCQAAAABJRU5ErkJggg==\n",
      "text/latex": [
       "$$\\left[\\begin{matrix}\\phi_{0}(x_1)^{2} + \\phi_{1}(x_1)^{2} + \\phi_{2}(x_1)^{2} & \\phi_{0}(x_1) \\phi_{0}(x_2) + \\phi_{1}(x_1) \\phi_{1}(x_2) + \\phi_{2}(x_1) \\phi_{2}(x_2)\\\\\\phi_{0}(x_1) \\phi_{0}(x_2) + \\phi_{1}(x_1) \\phi_{1}(x_2) + \\phi_{2}(x_1) \\phi_{2}(x_2) & \\phi_{0}(x_2)^{2} + \\phi_{1}(x_2)^{2} + \\phi_{2}(x_2)^{2}\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡                                2                2                2          \n",
       "⎢                   \\phi_{0}(x_1)  + \\phi_{1}(x_1)  + \\phi_{2}(x_1)           \n",
       "⎢                                                                             \n",
       "⎢                                                                             \n",
       "⎣\\phi_{0}(x_1)⋅\\phi_{0}(x_2) + \\phi_{1}(x_1)⋅\\phi_{1}(x_2) + \\phi_{2}(x_1)⋅\\ph\n",
       "\n",
       "                                                                              \n",
       "            \\phi_{0}(x_1)⋅\\phi_{0}(x_2) + \\phi_{1}(x_1)⋅\\phi_{1}(x_2) + \\phi_{\n",
       "                                                                              \n",
       "                                            2                2                \n",
       "i_{2}(x_2)                     \\phi_{0}(x_2)  + \\phi_{1}(x_2)  + \\phi_{2}(x_2)\n",
       "\n",
       "                     ⎤\n",
       "2}(x_1)⋅\\phi_{2}(x_2)⎥\n",
       "                     ⎥\n",
       "2                    ⎥\n",
       "                     ⎦"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "E2 = phi_0 @ phi_0.T + phi_1 @ phi_1.T + phi_2 @ phi_2.T\n",
    "E2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "E1 == E2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Geometrical Representation for least squares"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pseudoinverse(phi):\n",
    "    return inv(phi.T @ phi) @ phi.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.94 ,  0.044,  0.128, -0.099,  0.017,  0.061,  0.155],\n",
       "       [ 0.044,  0.803, -0.039,  0.023, -0.317, -0.229, -0.037],\n",
       "       [ 0.128, -0.039,  0.707,  0.228,  0.063, -0.07 , -0.358],\n",
       "       [-0.099,  0.023,  0.228,  0.822, -0.062,  0.046,  0.279],\n",
       "       [ 0.017, -0.317,  0.063, -0.062,  0.433, -0.357,  0.096],\n",
       "       [ 0.061, -0.229, -0.07 ,  0.046, -0.357,  0.733, -0.073],\n",
       "       [ 0.155, -0.037, -0.358,  0.279,  0.096, -0.073,  0.561]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N, M = 7, 5\n",
    "Phi = np.random.randn(N, M)\n",
    "Phi @ pseudoinverse(Phi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2],\n",
       "       [1],\n",
       "       [4],\n",
       "       [1],\n",
       "       [5],\n",
       "       [7],\n",
       "       [1]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = np.array([2, 1, 4, 1, 5, 7, 1]).reshape(-1, 1)\n",
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.006],\n",
       "       [-2.467],\n",
       "       [ 2.743],\n",
       "       [ 1.852],\n",
       "       [-0.328],\n",
       "       [ 2.932],\n",
       "       [-0.349]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Phi @ pseudoinverse(Phi) @ v"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiple Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "w11, w12, w21, w22 = sympy.symbols(\"w_{11}, w_{12}, w_{21}, w_{22}\")\n",
    "x1, x2 = sympy.symbols(\"x_1 x_2\")\n",
    "\n",
    "W = sympy.Matrix([\n",
    "    [w11, w12],\n",
    "    [w21, w22]\n",
    "])\n",
    "x = sympy.Matrix([[x1, x2]]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVIAAAAbBAMAAADc2xrMAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAIma7zZnddlTvRIkQqzLsm4+cAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAEKUlEQVRYCa1XS2gTURQ9kzSfZmIMiiJujKKuq+BOMWIQEZTid1GFCkL9LMzCImLFilBbECwoCCKYlSKCjWLpRnCEQhERFMRuu9CNm1Y3ilTrvfe9N3mTZNo0yYXp3HfvPWfOzGRmToG2hXvkZNu4qonGd3jVpRbWL/C5BfSiUDe7Ir/owPKapzGbXR6i4el4NvOj4eGlB89jQ2npqaYm4iXnT1PAMNAdL6zTcj053zKFTdBnL9qbx4vt5HO728kW5BqnpZOjP63GRibYUsOyqqay/EKEH9MIX4TVywfXIlYQXaqcmgl2nK7gurnVJ4I9w3NgT3P4ICqdB74O3/SC1Q6+HC3HJJDZP7xTXdfF2ZLFxfvcnQAeLSzouVt6X/tz0I3KrgFut4jOhYU5xJc+7wbY8K5ycMAoPW0X6+cNcKdHFXRtfQa72gAbOkoWQit1eq1aSNoI9wGF7QfS/UVsxojnk01dW3/VuuOGbeoEIvnkQ38sffnTE7rrEoE3lFaa6AaqMajPXSPB5uY3FMV2wE2OYhfGSvhKa9rS5UhfPI9MnvsURml5BNGPqT9gY8PN99haPANloSK09kMrjeYAg+F5xoVwGwlMJoQ292ZFfBZYF8vjJ6LZb18A3lzPnU9mM6d61YRR6gwexcoctrGxkeZrvPFuQFmopBlmjFYaH4TB8LwYohBuI4HJhNDmHlI6HgBeZ5GsSgzgEm1pdHRRpg7uFgr77hUKH7l8n91StxgbbnrYS2PKQmXmKJV4VSicKxQOUt5Z8jFMJrgQbiOByYTQ5p5VxKQUY9nEKFxfKbCySFWllBNeUaR+4ArdTzE2qvmLqspCVZRSSV9TUmowolQZovrcWgKTaU9W4dZK6e7jOP3+sM5SOjtD1RqlsV76OTuD1JlXzcw/GgPYQqV6OdNRufswGCETQ1Sf20gQMia0uB8rWnqi6PjxMsq+0qR3HQmvVimdzgW4NB8vSvN2Yg4DtOyjLZKnPya0Un6iDEaUMi6E20gQMiK0ufUTxa+AS3jalSn5SscGd4O8hZDzwc3dT+RTd/GdCoShZup3dM7J0QPYTaUYnagfWik3DEbICBfGbSQwhrYAd48i5ksb6xmYfkn7IbV9uDx1OFdHKY5NfLhKDTY2dOT09ED/JkLIN7NjhjITWqn8IjSGlTIujNtIYDLaAtyHFK/9NdVK9QHlMnBurqmui7Hxm9pCBT51Wim9FvzgeTFEquLDq7jZj1meTFKHoBx8oiZGKOFNBX0PVKSzOlE7MTZ+U1so862Skbd6Xq62ymlecLrjw6u42Y9ZnkxScigqtuk9sObitGyq4Bz/a0b8CUnY2FhNsVDs+moj6p8iz4shUkMWPIhisoonU+mkGWmLk65oMrS8d7rsVbO5fymdXLMUFm6jlVspvUFaDvnvhFj+A5x+T051lzssAAAAAElFTkSuQmCC\n",
      "text/latex": [
       "$$\\left(w_{11} x_{1} + w_{21} x_{2}\\right)^{2} + \\left(w_{12} x_{1} + w_{22} x_{2}\\right)^{2}$$"
      ],
      "text/plain": [
       "                       2                          2\n",
       "(w_{11}⋅x₁ + w_{21}⋅x₂)  + (w_{12}⋅x₁ + w_{22}⋅x₂) "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((W.T @ x).T @ (W.T @ x))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAAyBAMAAABPMGYoAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMA74lUMhB2qyJEZt3Nu5lcG7FTAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAE1klEQVRoBe2aT4gURxTGv51/zuzMrAMmuQXnokgIZhcxBELIQswhILHjYfU4CmsuweS0BPQw4EmFZAwh192jJCGYHMRAIAM55LKBvQQ9mSWggpqwJGo0l/G9+tP9pru66aVr0YUpmO7q96rr+6a6uqf6x2D36B9sz1IajTrYdejd7eke1UPvdPBCwvyFRGTzgctZp3gTqDjs19eypHPmdvTTG/oTcNlvZSine4plqsNYQBz6E3DZ/04opVRfT4mL8F5Rj1X9CbjsX4upOQ5z2P/AcZoJ+RNw2K9fSRe2mRz2WwPbOL73KGDsL3x4/mTPyNTWgYXbmB6WXwyFq6c+/UrOBmO/erqHbzEXhO1EPyXqJioiAY8C2n61O321OURlyHrtFaA7h/YnjScqwtGj+Kb3E3CY81yM/VK5g+tYHpT+uqWSpp/9N+kLTQ9VS72RCZ8C2n4pKD0s9yt31LRpzqI+ex87V3CDIyq6B68Fb+D9M9aRsX9uZoj/0e7vw1mV1P2U+jvIell1Zk6QCXgUMKOP1hoJacWpAap4CUf6WFcRjgb4jY28qdy8urr6/erqr1QPpnqVx5jBNW5Nyarqp9mnICobqrHeyAQ8Cpi5j5090gnto/EYJ2gKhfaBR+xD26eKGX0s92sdlPAjPh/oJPfTHNSfxOxDCegE2/clYO0fmSdT2j5dW8xcoTldn43sVx5QPmn/Ht0iOEeZtwOdVP2g/JAM0kUTJUrw5PEloO2Xg49RC4x9vnXJ1A80qqH9g7UNLDrsX0ezSxcJuKqSph80e7FbVybUretJQNtfnv0DtIzSo89PvNqw8TLeC+03/m1v1Fcc9j/CpbUKzQU+h2aW6Qf7QePbpY0tMqEaexLQ9o+dWri7Yu2rq/733mMnTYS+VHVp8fTXZCUx92f+XFx6hRK8DKCk6Wd6nQ5b87SxRSb0tPIjoO0bFT369BwJC0dMlGMJ+6Zho9uYj5L4Al8CF8NOoopKwKPAuP2OUhIrKvpZAn9MmbOV47ai94cPvBUgTFZ+OfA7IH+kTWud0NfKhAoKSPv1e//1uNd2n7dcOGKjdPjZz0sqnNjsHo1Ecmo02oBrwawSdLI/AWnfuvLyNhFZtN1Ge38CLvv8ECpcLmf14E3AaT9L+fnKTew/y+sxGf3nbvS9PRjSvpk3Adfk8fJYnmCqtGun4+pn3TX6Ys2T1oN920rLU9yx5rGt/Qm47IsVpxWM73PYn2Cq+KCNHzMHM6O/RRQp0tsiAW0/E1MxdMqBqRhVSRqVH1MVEND2szCVgk45MBWjKkmjxEsavQsrDqb5VRxTFREwo5+BqRR0yoGpGFVJGpUbUxURMHM/A1MZtvSI53HiXVdgKoWqBI0as5+FqYoIWPuKIunrHadICjrlwFSMqhDRqPyYqoCAtp+JqRg65cFUhKokjRq7dWUigakKCGj7mZiKoFMuTEWoStKo/JiqgIC2n4mpCDrlwlS0FJA0qjUfPfXHEglMVUDAzn0lZZ51YtHAEcOWuEXi1jUOGVMpVKWPN4mpCgiM29dASqyoiCJZtsTOQhLlwFQKVSn7m8RURQSkfQukIkTDEcuWyFkmpmJUpcsmMVURAWnfqMPL60o0BrbbaO9PwGV/gqmikd7aGo3+Nv9DzPb+O9JT0Fzgmbf7t/0AAAAASUVORK5CYII=\n",
      "text/latex": [
       "$$\\left[\\begin{matrix}x_{1} \\left(w_{11} x_{1} + w_{21} x_{2}\\right) & x_{2} \\left(w_{11} x_{1} + w_{21} x_{2}\\right)\\\\x_{1} \\left(w_{12} x_{1} + w_{22} x_{2}\\right) & x_{2} \\left(w_{12} x_{1} + w_{22} x_{2}\\right)\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡x₁⋅(w_{11}⋅x₁ + w_{21}⋅x₂)  x₂⋅(w_{11}⋅x₁ + w_{21}⋅x₂)⎤\n",
       "⎢                                                      ⎥\n",
       "⎣x₁⋅(w_{12}⋅x₁ + w_{22}⋅x₂)  x₂⋅(w_{12}⋅x₁ + w_{22}⋅x₂)⎦"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(W.T @ x) @ x.T"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
